Libraries
suppressPackageStartupMessages({
library(here)
library(ggplot2)
library(gplots)
library(LSD)
library(pander)
library(pracma)
library(RColorBrewer)
library(viridis)
})
Palette
hpal <- colorRampPalette(c("blue","white","red"))(100)
cols <- brewer.pal(12,"Paired")
mar <- par("mar")
AUCs <- sapply(dir(here("data/seidr/roc"),
pattern=".*_roc.tsv$",full.names=TRUE),
function(f){
algo <- sub("[^_]+_","",sub("_roc\\.tsv","",f))
dat <- read.delim(f,header=FALSE,skip=1,col.names = c("TP","FP","PR"))
head <- scan(f,nmax = 3,what = "character",sep="\t")
message(sprintf("Found %s GS edges out of %s edges",head[2],sum(as.integer(head[2:3]))))
auc <- round(trapz(dat[,2],dat[,1]),digits=3)
plot(dat[,2],dat[,1],type="l",main=sprintf("%s (AUC = %s)",algo,auc),
xlab="False Positive Rate",ylab="True Positive Rate",
sub=sprintf("%s Gold Standard edges out of %s edges\n%s",head[2],sum(as.integer(head[2:3])),
basename(f)))
abline(0,1,lty=2)
message(sprintf("The AUC is %s",auc))
return(c(sub("_.*","",basename(f)),algo,auc))
})
## Found 79 GS edges out of 30856 edges
## The AUC is 0.454
## Found 215792 GS edges out of 240236315 edges
## The AUC is 0.567
## Found 164057 GS edges out of 166252177 edges
## The AUC is 0.619
## Found 146353 GS edges out of 201569549 edges
## The AUC is 0.601
## Found 222561 GS edges out of 464865786 edges
## The AUC is 0.763
## Found 3179 GS edges out of 48894954 edges
## The AUC is 0.236
## Found 222561 GS edges out of 464865786 edges
## The AUC is 0.763
## Found 2541 GS edges out of 26431075 edges
## The AUC is 0.847
## Found 222561 GS edges out of 464865786 edges
## The AUC is 0.66
## Found 222561 GS edges out of 464865786 edges
## The AUC is 0.788
## Found 222559 GS edges out of 464860283 edges
## The AUC is 0.614
## Found 222561 GS edges out of 464863253 edges
## The AUC is 0.721
## Found 33621 GS edges out of 21936947 edges
## The AUC is 0.586
## Found 5 GS edges out of 3737 edges
## The AUC is 0.631
## Found 7949 GS edges out of 1386334 edges
## The AUC is 0.47
## Found 7953 GS edges out of 1324170 edges
## The AUC is 0.812
## Found 7903 GS edges out of 1334447 edges
## The AUC is 0.678
## Found 7965 GS edges out of 1390175 edges
## The AUC is 0.739
## Found 125 GS edges out of 641301 edges
## The AUC is 0.523
## Found 7965 GS edges out of 1390175 edges
## The AUC is 0.529
## Found 994 GS edges out of 166099 edges
## The AUC is 0.788
## Found 7965 GS edges out of 1390175 edges
## The AUC is 0.674
## Found 7965 GS edges out of 1390175 edges
## The AUC is 0.73
## Found 7965 GS edges out of 1390155 edges
## The AUC is 0.498
## Found 7965 GS edges out of 1390175 edges
## The AUC is 0.827
## Found 7461 GS edges out of 1338013 edges
## The AUC is 0.514
## Found 1 GS edges out of 1757 edges
## The AUC is 0.06
## Found 854 GS edges out of 63605 edges
## The AUC is 0.473
## Found 854 GS edges out of 62361 edges
## The AUC is 0.766
## Found 854 GS edges out of 63177 edges
## The AUC is 0.592
## Found 855 GS edges out of 63656 edges
## The AUC is 0.757
## Found 9 GS edges out of 16347 edges
## The AUC is 0.431
## Found 855 GS edges out of 63656 edges
## The AUC is 0.667
## Found 495 GS edges out of 27719 edges
## The AUC is 0.779
## Found 855 GS edges out of 63656 edges
## The AUC is 0.545
## Found 855 GS edges out of 63656 edges
## The AUC is 0.736
## Found 855 GS edges out of 63654 edges
## The AUC is 0.344
## Found 855 GS edges out of 63656 edges
## The AUC is 0.758
## Found 851 GS edges out of 63086 edges
## The AUC is 0.604
## Found 1 GS edges out of 2031 edges
## The AUC is 0.054
## Found 1656 GS edges out of 129175 edges
## The AUC is 0.504
## Found 1658 GS edges out of 126172 edges
## The AUC is 0.789
## Found 1658 GS edges out of 128011 edges
## The AUC is 0.611
## Found 1659 GS edges out of 129323 edges
## The AUC is 0.74
## Found 20 GS edges out of 40283 edges
## The AUC is 0.436
## Found 1659 GS edges out of 129323 edges
## The AUC is 0.653
## Found 622 GS edges out of 45838 edges
## The AUC is 0.783
## Found 1659 GS edges out of 129323 edges
## The AUC is 0.548
## Found 1659 GS edges out of 129323 edges
## The AUC is 0.746
## Found 1659 GS edges out of 129320 edges
## The AUC is 0.351
## Found 1659 GS edges out of 129323 edges
## The AUC is 0.79
## Found 1639 GS edges out of 127703 edges
## The AUC is 0.575
## Found 1 GS edges out of 2261 edges
## The AUC is 0.049
## Found 2439 GS edges out of 206779 edges
## The AUC is 0.507
## Found 2443 GS edges out of 201466 edges
## The AUC is 0.794
## Found 2442 GS edges out of 204315 edges
## The AUC is 0.628
## Found 2444 GS edges out of 207073 edges
## The AUC is 0.731
## Found 34 GS edges out of 71418 edges
## The AUC is 0.532
## Found 2444 GS edges out of 207073 edges
## The AUC is 0.63
## Found 712 GS edges out of 61958 edges
## The AUC is 0.778
## Found 2444 GS edges out of 207073 edges
## The AUC is 0.569
## Found 2444 GS edges out of 207073 edges
## The AUC is 0.742
## Found 2444 GS edges out of 207070 edges
## The AUC is 0.375
## Found 2444 GS edges out of 207073 edges
## The AUC is 0.801
## Found 2406 GS edges out of 204022 edges
## The AUC is 0.55
## Found 2 GS edges out of 2443 edges
## The AUC is 0.492
## Found 3206 GS edges out of 302715 edges
## The AUC is 0.505
## Found 3209 GS edges out of 294143 edges
## The AUC is 0.804
## Found 3206 GS edges out of 298155 edges
## The AUC is 0.641
## Found 3211 GS edges out of 303241 edges
## The AUC is 0.732
## Found 44 GS edges out of 112782 edges
## The AUC is 0.588
## Found 3211 GS edges out of 303241 edges
## The AUC is 0.61
## Found 764 GS edges out of 77665 edges
## The AUC is 0.782
## Found 3211 GS edges out of 303241 edges
## The AUC is 0.59
## Found 3211 GS edges out of 303241 edges
## The AUC is 0.741
## Found 3211 GS edges out of 303237 edges
## The AUC is 0.395
## Found 3211 GS edges out of 303241 edges
## The AUC is 0.809
## Found 3155 GS edges out of 297960 edges
## The AUC is 0.536
## Found 4 GS edges out of 2656 edges
## The AUC is 0.69
## Found 4035 GS edges out of 423679 edges
## The AUC is 0.501
## Found 4036 GS edges out of 410422 edges
## The AUC is 0.809
## Found 4033 GS edges out of 415791 edges
## The AUC is 0.655
## Found 4041 GS edges out of 424495 edges
## The AUC is 0.734
## Found 58 GS edges out of 167359 edges
## The AUC is 0.589
## Found 4041 GS edges out of 424495 edges
## The AUC is 0.593
## Found 806 GS edges out of 93594 edges
## The AUC is 0.785
## Found 4041 GS edges out of 424495 edges
## The AUC is 0.61
## Found 4041 GS edges out of 424495 edges
## The AUC is 0.74
## Found 4041 GS edges out of 424487 edges
## The AUC is 0.417
## Found 4041 GS edges out of 424495 edges
## The AUC is 0.817
## Found 3966 GS edges out of 415989 edges
## The AUC is 0.522
## Found 4 GS edges out of 2828 edges
## The AUC is 0.674
## Found 4826 GS edges out of 562321 edges
## The AUC is 0.496
## Found 4826 GS edges out of 543347 edges
## The AUC is 0.812
## Found 4817 GS edges out of 549872 edges
## The AUC is 0.66
## Found 4833 GS edges out of 563494 edges
## The AUC is 0.735
## Found 75 GS edges out of 232132 edges
## The AUC is 0.55
## Found 4833 GS edges out of 563494 edges
## The AUC is 0.578
## Found 852 GS edges out of 108533 edges
## The AUC is 0.787
## Found 4833 GS edges out of 563494 edges
## The AUC is 0.625
## Found 4833 GS edges out of 563494 edges
## The AUC is 0.739
## Found 4833 GS edges out of 563483 edges
## The AUC is 0.435
## Found 4833 GS edges out of 563494 edges
## The AUC is 0.823
## Found 4713 GS edges out of 550807 edges
## The AUC is 0.512
## Found 4 GS edges out of 3019 edges
## The AUC is 0.656
## Found 5574 GS edges out of 705886 edges
## The AUC is 0.49
## Found 5575 GS edges out of 680493 edges
## The AUC is 0.813
## Found 5559 GS edges out of 688064 edges
## The AUC is 0.665
## Found 5582 GS edges out of 707511 edges
## The AUC is 0.735
## Found 84 GS edges out of 301117 edges
## The AUC is 0.553
## Found 5582 GS edges out of 707511 edges
## The AUC is 0.566
## Found 892 GS edges out of 121657 edges
## The AUC is 0.788
## Found 5582 GS edges out of 707511 edges
## The AUC is 0.638
## Found 5582 GS edges out of 707511 edges
## The AUC is 0.738
## Found 5582 GS edges out of 707498 edges
## The AUC is 0.45
## Found 5582 GS edges out of 707511 edges
## The AUC is 0.824
## Found 5410 GS edges out of 689674 edges
## The AUC is 0.506
## Found 5 GS edges out of 3239 edges
## The AUC is 0.672
## Found 6326 GS edges out of 890325 edges
## The AUC is 0.484
## Found 6325 GS edges out of 855698 edges
## The AUC is 0.813
## Found 6304 GS edges out of 864523 edges
## The AUC is 0.671
## Found 6335 GS edges out of 892514 edges
## The AUC is 0.738
## Found 93 GS edges out of 391311 edges
## The AUC is 0.553
## Found 6335 GS edges out of 892514 edges
## The AUC is 0.553
## Found 930 GS edges out of 135963 edges
## The AUC is 0.787
## Found 6335 GS edges out of 892514 edges
## The AUC is 0.651
## Found 6335 GS edges out of 892514 edges
## The AUC is 0.735
## Found 6335 GS edges out of 892497 edges
## The AUC is 0.467
## Found 6335 GS edges out of 892514 edges
## The AUC is 0.827
## Found 6083 GS edges out of 866956 edges
## The AUC is 0.506
## Found 5 GS edges out of 3493 edges
## The AUC is 0.651
## Found 7132 GS edges out of 1127869 edges
## The AUC is 0.476
## Found 7133 GS edges out of 1080524 edges
## The AUC is 0.812
## Found 7100 GS edges out of 1090450 edges
## The AUC is 0.675
## Found 7144 GS edges out of 1130829 edges
## The AUC is 0.74
## Found 110 GS edges out of 510463 edges
## The AUC is 0.543
## Found 7144 GS edges out of 1130829 edges
## The AUC is 0.539
## Found 968 GS edges out of 151492 edges
## The AUC is 0.786
## Found 7144 GS edges out of 1130829 edges
## The AUC is 0.664
## Found 7144 GS edges out of 1130829 edges
## The AUC is 0.733
## Found 7144 GS edges out of 1130811 edges
## The AUC is 0.485
## Found 7144 GS edges out of 1130829 edges
## The AUC is 0.829
## Found 6778 GS edges out of 1093491 edges
## The AUC is 0.509
## Found 19 GS edges out of 11218 edges
## The AUC is 0.401
## Found 10324 GS edges out of 1759138 edges
## The AUC is 0.259
## Found 10315 GS edges out of 1636546 edges
## The AUC is 0.81
## Found 10207 GS edges out of 1635291 edges
## The AUC is 0.639
## Found 10330 GS edges out of 1760498 edges
## The AUC is 0.665
## Found 236 GS edges out of 884754 edges
## The AUC is 0.576
## Found 10330 GS edges out of 1760498 edges
## The AUC is 0.293
## Found 1070 GS edges out of 176869 edges
## The AUC is 0.757
## Found 10330 GS edges out of 1760498 edges
## The AUC is 0.702
## Found 10330 GS edges out of 1760498 edges
## The AUC is 0.662
## Found 10330 GS edges out of 1760482 edges
## The AUC is 0.618
## Found 10330 GS edges out of 1760498 edges
## The AUC is 0.841
## Found 8510 GS edges out of 1529623 edges
## The AUC is 0.551
tab <- split.data.frame(t(AUCs[2:3,]),AUCs[1,])
stopifnot(nrow(unique(t(sapply(tab,function(ta){ta[,1]})))) == 1)
res <- sapply(tab,function(ta){as.numeric(ta[,2])})
rownames(res) <- tab[[1]][,1]
res <- res[,c(3:(ncol(res)-1),2,1,ncol(res))]
Re-order the data.
rocord <- c("backbone-1-percent", "backbone-2-percent", "backbone-3-percent",
"backbone-4-percent", "backbone-5-percent", "backbone-6-percent",
"backbone-7-percent", "backbone-8-percent", "backbone-9-percent",
"backbone-10-percent", "aggregated","hard-threshold-dot3")
Create a heatmap of the AUC values for each network and for each algorithm.
heatmap.2(res,
trace="none",
margins = c(10.1,7.1),
col = hpal)
heatmap.2(as.matrix(res[,rocord]),
Colv = FALSE,
dendrogram = "row",
trace="none",
margins = c(10.1,7.1),
col = hpal)
Obtain a line-plot from this heatmap.
par(mar=c(10.1,3.1,3.1,0.1))
linesplot(res[,rocord],cols = colorRampPalette(c("blue","red"))(20),
addboxes = TRUE,las=2,main="AUC distribution")
par(mar=mar)
par(mar=c(10.1,4.1,3.1,2.1))
plot(0,0,ylim=c(0,1),
xlim=c(1,ncol(res)),type="n",
xaxt="n",xlab="",ylab="AUC",main="AUC evolution")
axis(1,1:ncol(res),las=2,labels=colnames(res))
sapply(1:nrow(res),function(i){lines(res[i,],col=cols[i],lwd=2,lty=2)})
## [[1]]
## NULL
##
## [[2]]
## NULL
##
## [[3]]
## NULL
##
## [[4]]
## NULL
##
## [[5]]
## NULL
##
## [[6]]
## NULL
##
## [[7]]
## NULL
##
## [[8]]
## NULL
##
## [[9]]
## NULL
##
## [[10]]
## NULL
##
## [[11]]
## NULL
##
## [[12]]
## NULL
##
## [[13]]
## NULL
legend("bottomright",legend = rownames(res),col=cols[1:nrow(res)],lty=1,lwd=2)
par(mar=mar)
pander(res)
| backbone-1-percent | backbone-2-percent | backbone-3-percent | |
|---|---|---|---|
| ARACNE | 0.06 | 0.054 | 0.049 |
| CLR | 0.473 | 0.504 | 0.507 |
| ELNET | 0.766 | 0.789 | 0.794 |
| GENIE3 | 0.592 | 0.611 | 0.628 |
| irp | 0.757 | 0.74 | 0.731 |
| LLR | 0.431 | 0.436 | 0.532 |
| MI | 0.667 | 0.653 | 0.63 |
| NARROMI | 0.779 | 0.783 | 0.778 |
| PCOR | 0.545 | 0.548 | 0.569 |
| PEARSON | 0.736 | 0.746 | 0.742 |
| PLSNET | 0.344 | 0.351 | 0.375 |
| SPEARMAN | 0.758 | 0.79 | 0.801 |
| TIGRESS | 0.604 | 0.575 | 0.55 |
| backbone-4-percent | backbone-5-percent | backbone-6-percent | |
|---|---|---|---|
| ARACNE | 0.492 | 0.69 | 0.674 |
| CLR | 0.505 | 0.501 | 0.496 |
| ELNET | 0.804 | 0.809 | 0.812 |
| GENIE3 | 0.641 | 0.655 | 0.66 |
| irp | 0.732 | 0.734 | 0.735 |
| LLR | 0.588 | 0.589 | 0.55 |
| MI | 0.61 | 0.593 | 0.578 |
| NARROMI | 0.782 | 0.785 | 0.787 |
| PCOR | 0.59 | 0.61 | 0.625 |
| PEARSON | 0.741 | 0.74 | 0.739 |
| PLSNET | 0.395 | 0.417 | 0.435 |
| SPEARMAN | 0.809 | 0.817 | 0.823 |
| TIGRESS | 0.536 | 0.522 | 0.512 |
| backbone-7-percent | backbone-8-percent | backbone-9-percent | |
|---|---|---|---|
| ARACNE | 0.656 | 0.672 | 0.651 |
| CLR | 0.49 | 0.484 | 0.476 |
| ELNET | 0.813 | 0.813 | 0.812 |
| GENIE3 | 0.665 | 0.671 | 0.675 |
| irp | 0.735 | 0.738 | 0.74 |
| LLR | 0.553 | 0.553 | 0.543 |
| MI | 0.566 | 0.553 | 0.539 |
| NARROMI | 0.788 | 0.787 | 0.786 |
| PCOR | 0.638 | 0.651 | 0.664 |
| PEARSON | 0.738 | 0.735 | 0.733 |
| PLSNET | 0.45 | 0.467 | 0.485 |
| SPEARMAN | 0.824 | 0.827 | 0.829 |
| TIGRESS | 0.506 | 0.506 | 0.509 |
| backbone-10-percent | aggregated | hard-threshold-dot3 | |
|---|---|---|---|
| ARACNE | 0.631 | 0.454 | 0.401 |
| CLR | 0.47 | 0.567 | 0.259 |
| ELNET | 0.812 | 0.619 | 0.81 |
| GENIE3 | 0.678 | 0.601 | 0.639 |
| irp | 0.739 | 0.763 | 0.665 |
| LLR | 0.523 | 0.236 | 0.576 |
| MI | 0.529 | 0.763 | 0.293 |
| NARROMI | 0.788 | 0.847 | 0.757 |
| PCOR | 0.674 | 0.66 | 0.702 |
| PEARSON | 0.73 | 0.788 | 0.662 |
| PLSNET | 0.498 | 0.614 | 0.618 |
| SPEARMAN | 0.827 | 0.721 | 0.841 |
| TIGRESS | 0.514 | 0.586 | 0.551 |
write.csv(res[,rocord],file=here("data/seidr/roc/AUCs.csv"),quote=FALSE)
## R version 3.6.1 (2019-07-05)
## Platform: x86_64-pc-linux-gnu (64-bit)
## Running under: Ubuntu 18.04.3 LTS
##
## Matrix products: default
## BLAS/LAPACK: /usr/lib/x86_64-linux-gnu/libopenblasp-r0.2.20.so
##
## locale:
## [1] LC_CTYPE=en_US.UTF-8 LC_NUMERIC=C
## [3] LC_TIME=en_US.UTF-8 LC_COLLATE=en_US.UTF-8
## [5] LC_MONETARY=en_US.UTF-8 LC_MESSAGES=en_US.UTF-8
## [7] LC_PAPER=en_US.UTF-8 LC_NAME=C
## [9] LC_ADDRESS=C LC_TELEPHONE=C
## [11] LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C
##
## attached base packages:
## [1] stats graphics grDevices utils datasets methods base
##
## other attached packages:
## [1] viridis_0.5.1 viridisLite_0.3.0 RColorBrewer_1.1-2 pracma_2.2.5
## [5] pander_0.6.3 LSD_4.0-0 gplots_3.0.1.1 ggplot2_3.2.1
## [9] here_0.1
##
## loaded via a namespace (and not attached):
## [1] Rcpp_1.0.3 pillar_1.4.2 compiler_3.6.1 highr_0.8
## [5] bitops_1.0-6 tools_3.6.1 digest_0.6.23 evaluate_0.14
## [9] lifecycle_0.1.0 tibble_2.1.3 gtable_0.3.0 pkgconfig_2.0.3
## [13] rlang_0.4.2 yaml_2.2.0 xfun_0.11 gridExtra_2.3
## [17] withr_2.1.2 stringr_1.4.0 dplyr_0.8.3 knitr_1.26
## [21] gtools_3.8.1 caTools_1.17.1.2 rprojroot_1.3-2 grid_3.6.1
## [25] tidyselect_0.2.5 glue_1.3.1 R6_2.4.1 rmarkdown_1.18
## [29] gdata_2.18.0 purrr_0.3.3 magrittr_1.5 backports_1.1.5
## [33] scales_1.1.0 htmltools_0.4.0 assertthat_0.2.1 colorspace_1.4-1
## [37] KernSmooth_2.23-16 stringi_1.4.3 lazyeval_0.2.2 munsell_0.5.0
## [41] crayon_1.3.4